{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[AMP WARNING][Frontend.cpp:121][1657891302:36191]Sleep 0.1s waiting for AMP Server socket: /tmp/harp/ccl_ipc_socket/system/server.socket\n",
      "[AMP INFO][Frontend.cpp:152][1657891302:136464]pid=232, start to allocate gpu resource ...\n",
      "Reading lines...\n",
      "Load 65080 sentence pairs\n",
      "Counting words...\n",
      "['will', 'you', 'come', 'round', 'next', 'tuesday', '?'] ['next', 'tuesday', '?', 'just', 'a', 'moment', '.', 'it', 'will', 'be', 'the', 'twenty-first', 'of', 'may', '?', 'oh', ',', 'dear', '.', 'hum', ',', 'i', 'promised', 'my', 'aunt', 'i', '’', 'll', 'be', 'at', 'her', 'birthday', 'party', '.']\n",
      "Counted words:\n",
      "dialog 21609\n",
      "(tensor([126,  28,  26, 597, 191, 669,  14,   3,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1]), tensor(8), tensor([  191,   669,    14,   173,    33,   864,    25,    69,   126,   105,\n",
      "           43, 15610,    23,   255,    14,   160,     6,  2698,    25,  2919,\n",
      "            6,    15,  3217,   215,  5543]), tensor(25))\n",
      "Reading lines...\n",
      "Load 8975 sentence pairs\n",
      "Reading lines...\n",
      "Load 8974 sentence pairs\n"
     ]
    }
   ],
   "source": [
    "!python data.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "from config import *\n",
    "from transformer import TransformerEncoder, TransformerDecoder, EncoderDecoder\n",
    "from attention import sequence_mask\n",
    "from data import loaddata, MyDataset, Lang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Timer:\n",
    "    \"\"\"Record multiple running times.\"\"\"\n",
    "    def __init__(self):\n",
    "        self.times = []\n",
    "        self.start()\n",
    "\n",
    "    def start(self):\n",
    "        \"\"\"Start the timer.\"\"\"\n",
    "        self.tik = time.time()\n",
    "\n",
    "    def stop(self):\n",
    "        \"\"\"Stop the timer and record the time in a list.\"\"\"\n",
    "        self.times.append(time.time() - self.tik)\n",
    "        return self.times[-1]\n",
    "\n",
    "    def avg(self):\n",
    "        \"\"\"Return the average time.\"\"\"\n",
    "        return sum(self.times) / len(self.times)\n",
    "\n",
    "    def sum(self):\n",
    "        \"\"\"Return the sum of time.\"\"\"\n",
    "        return sum(self.times)\n",
    "\n",
    "    def cumsum(self):\n",
    "        \"\"\"Return the accumulated time.\"\"\"\n",
    "        return np.array(self.times).cumsum().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import time\n",
    "from IPython import display\n",
    "    \n",
    "def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):\n",
    "    \"\"\"Set the axes for matplotlib.\"\"\"\n",
    "    axes.set_xlabel(xlabel)\n",
    "    axes.set_ylabel(ylabel)\n",
    "    axes.set_xscale(xscale)\n",
    "    axes.set_yscale(yscale)\n",
    "    axes.set_xlim(xlim)\n",
    "    axes.set_ylim(ylim)\n",
    "    if legend:\n",
    "        axes.legend(legend)\n",
    "    axes.grid()\n",
    "\n",
    "class Animator:\n",
    "    \"\"\"For plotting data in animation.\"\"\"\n",
    "    def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,\n",
    "                 ylim=None, xscale='linear', yscale='linear',\n",
    "                 fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,\n",
    "                 figsize=(3.5, 2.5)):\n",
    "        # Incrementally plot multiple lines\n",
    "        if legend is None:\n",
    "            legend = []\n",
    "#         use_svg_display()\n",
    "        self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)\n",
    "        if nrows * ncols == 1:\n",
    "            self.axes = [self.axes,]\n",
    "        # Use a lambda function to capture arguments\n",
    "        self.config_axes = lambda: set_axes(self.axes[\n",
    "            0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)\n",
    "        self.X, self.Y, self.fmts = None, None, fmts\n",
    "\n",
    "    def add(self, x, y):\n",
    "        # Add multiple data points into the figure\n",
    "        if not hasattr(y, \"__len__\"):\n",
    "            y = [y]\n",
    "        n = len(y)\n",
    "        if not hasattr(x, \"__len__\"):\n",
    "            x = [x] * n\n",
    "        if not self.X:\n",
    "            self.X = [[] for _ in range(n)]\n",
    "        if not self.Y:\n",
    "            self.Y = [[] for _ in range(n)]\n",
    "        for i, (a, b) in enumerate(zip(x, y)):\n",
    "            if a is not None and b is not None:\n",
    "                self.X[i].append(a)\n",
    "                self.Y[i].append(b)\n",
    "        self.axes[0].cla()\n",
    "        for x, y, fmt in zip(self.X, self.Y, self.fmts):\n",
    "            self.axes[0].plot(x, y, fmt)\n",
    "        self.config_axes()\n",
    "        display.display(self.fig)\n",
    "        display.clear_output(wait=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "def loss_fn(out, tar):\n",
    "    out = out.contiguous().view(-1, out.shape[-1])\n",
    "    tar = tar.contiguous().view(-1)\n",
    "    return F.cross_entropy(out, tar, ignore_index=2)  # pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_seq2seq(net, data_iter, valid_iter, lr, num_epochs, lang, device, first_train = True, min_ppl=1e9):\n",
    "   # \"\"\"训练序列到序列模型\"\"\"\n",
    "    def xavier_init_weights(m):\n",
    "        if type(m) == nn.Linear:\n",
    "            nn.init.xavier_uniform_(m.weight)\n",
    "        if type(m) == nn.GRU:\n",
    "            for param in m._flat_weights_names:\n",
    "                if \"weight\" in param:\n",
    "                    nn.init.xavier_uniform_(m._parameters[param])\n",
    "    \n",
    "    net.to(device)\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n",
    "    net.train()\n",
    "    animator = Animator(xlabel='epoch', ylabel='loss',\n",
    "                     xlim=[0, num_epochs])\n",
    "    animator1 = Animator(xlabel='epoch', ylabel='ppl',\n",
    "                     xlim=[0, num_epochs])\n",
    "    LOSS = PPL = 0\n",
    "    \n",
    "    timer = Timer()\n",
    "    \n",
    "    if first_train:\n",
    "        net.apply(xavier_init_weights)\n",
    "    else:\n",
    "        net.eval()\n",
    "        PPL_AVG = 0\n",
    "        LOSS_AVG = 0\n",
    "        with torch.no_grad():\n",
    "#             i = 0\n",
    "            for batch in valid_iter:\n",
    "                # 转device\n",
    "                X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "                # inputs带<eos>不带<bos>，outputs带<bos>不带<eos>\n",
    "                # 初始bos\n",
    "                bos = torch.tensor([2  # lang.word2index['<bos>']\n",
    "                                    ] * Y.shape[0],\n",
    "                                   device=device).reshape(-1, 1)\n",
    "                dec_input = torch.cat([bos, Y[:, :-1]], 1)\n",
    "                # Teacher forcing, 输入X和正确答案，希望得到正确答案\n",
    "                # Y_hat\n",
    "                Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "                # Y_hat和Y中都有eos，没有bos\n",
    "                l = loss_fn(Y_hat, Y)\n",
    "                ppl = torch.exp(l)\n",
    "                LOSS_AVG += l.item()\n",
    "                PPL_AVG += ppl.item()\n",
    "            PPL_AVG /= len(valid_iter)\n",
    "            min_ppl = PPL_AVG\n",
    "            LOSS_AVG /= len(valid_iter)\n",
    "            animator.add(0, (LOSS_AVG, LOSS_AVG,))\n",
    "            animator1.add(0, (PPL_AVG, PPL_AVG,))\n",
    "                \n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        net.train()\n",
    "#         i = 0\n",
    "        for batch in data_iter:\n",
    "            optimizer.zero_grad()\n",
    "            X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "            bos = torch.tensor([2] * Y.shape[0],\n",
    "                          device=device).reshape(-1, 1)\n",
    "            dec_input = torch.cat([bos, Y[:, :-1]], 1)  # 强制教学\n",
    "            Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "            l = loss_fn(Y_hat, Y)\n",
    "            ppl = torch.exp(l)\n",
    "            l.backward()\n",
    "            # d2l.grad_clipping(net, 1)\n",
    "            optimizer.step()\n",
    "            with torch.no_grad():\n",
    "                LOSS += l.item()\n",
    "                PPL += ppl.item()\n",
    "#             i += 1\n",
    "#             if i == 30:\n",
    "#                 break\n",
    "        LOSS /= len(data_iter)\n",
    "        PPL /= len(data_iter)\n",
    "\n",
    "        \n",
    "        net.eval()\n",
    "        PPL_AVG = 0\n",
    "        LOSS_AVG = 0\n",
    "        with torch.no_grad():\n",
    "#             i = 0\n",
    "            for batch in valid_iter:\n",
    "                # 转device\n",
    "                X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "                # inputs带<eos>不带<bos>，outputs带<bos>不带<eos>\n",
    "                # 初始bos\n",
    "                bos = torch.tensor([2  # lang.word2index['<bos>']\n",
    "                                    ] * Y.shape[0],\n",
    "                                   device=device).reshape(-1, 1)\n",
    "                dec_input = torch.cat([bos, Y[:, :-1]], 1)\n",
    "                # Teacher forcing, 输入X和正确答案，希望得到正确答案\n",
    "                # Y_hat\n",
    "                Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "                # Y_hat和Y中都有eos，没有bos\n",
    "                l = loss_fn(Y_hat, Y)\n",
    "                ppl = torch.exp(l)\n",
    "                LOSS_AVG += l.item()\n",
    "                PPL_AVG += ppl.item()\n",
    "#                 i += 1\n",
    "#                 if i == 10:\n",
    "#                     break\n",
    "                    \n",
    "            PPL_AVG /= len(valid_iter)\n",
    "            LOSS_AVG /= len(valid_iter)\n",
    "            if PPL_AVG < min_ppl:\n",
    "                if min_ppl != 1e9:\n",
    "                    torch.save(encoder, MODEL_ROOT + \"trans_encoder1.mdl\")\n",
    "                    torch.save(decoder, MODEL_ROOT + \"trans_decoder1.mdl\")\n",
    "                min_ppl = PPL_AVG\n",
    "                \n",
    "        if (epoch + 1) % 2 == 0:\n",
    "            animator.add(epoch + 1, (LOSS, LOSS_AVG,))\n",
    "            animator1.add(epoch + 1, (PPL, PPL_AVG,))\n",
    "            \n",
    "    print(f'loss {LOSS:.3f}, PPL {PPL:.3f}, {timer.stop()/60:.1f} '\n",
    "        f'min on {str(device)}')\n",
    "    print(f'loss {LOSS_AVG:.3f}, PPL {PPL_AVG:.3f} ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = torch.load(DATA_ROOT + \"dataset\")\n",
    "lang = torch.load(DATA_ROOT + \"dialog.lang\")\n",
    "valid_data = torch.load(DATA_ROOT + \"validdata\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter = loaddata(dataset, 64)\n",
    "valid_iter = loaddata(valid_data, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = TransformerEncoder(\n",
    "        len(lang), key_size, query_size, value_size, num_hiddens,\n",
    "        norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,\n",
    "        num_layers, dropout)\n",
    "decoder = TransformerDecoder(\n",
    "        len(lang), key_size, query_size, value_size, num_hiddens,\n",
    "        norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,\n",
    "        num_layers, dropout)\n",
    "net = EncoderDecoder(encoder, decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = torch.load(MODEL_ROOT + \"trans_encoder{}.mdl\".format(num_examples))\n",
    "decoder = torch.load(MODEL_ROOT + \"trans_decoder{}.mdl\".format(num_examples))\n",
    "net = EncoderDecoder(encoder, decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.002"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 0.113, PPL 1.383, 2.8 min on cuda:0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAC4CAYAAAAsVToeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAFwlJREFUeJzt3Xt82/V97/HXR5Il2bJlRbZiyw7GCSEOMW0JyzkL0JSku5RlW9cyOD3rZS2jZdsppY/s0ctZT2lpu3XryrqywtrB1lLW26BjvTxG1532YGAUOCfh0pZA0txISGI78UWyfJEs6XP++ElGzlWOLcvK7/N8PPzwT7+L/Plaev++X/3000+iqhhj3MdT7QKMMdVh4TfGpSz8xriUhd8Yl7LwG+NSFn5jXMrCb4xLWfiNcSkLvzEuZeE3xqV81S5gPiKRiK5evbraZczL+Pg4oVCo2mXMi7VhadixY8dxVY2Vu35Nh7+trY3t27dXu4x56evrY/PmzdUuY16sDUuDiLw0l/Vt2G+MS1n4jXEpC78xLlXTr/mNqQRVJdOfYWLnBOPPjzO+c5yJnROsvW8t9d311S5vwVj4F1A+m2fi+QmSTyZJv5ym9c2tNF3eVO2yzGmoKpkjGcZ3jsN3YSI+QUNPA8f/9TjP/+7zM+v5lvkI9YbIJXMAHLz9ILlUjgu2XYCvuXYjVLuVLwHTw9PUResA2POBPRz50hHy4/mZ5ce/e5wNz21ARMhP5/HU1darrGwqS/pQmtxYjmwySy6ZIzeWI/qbUfytfhKPJxj4+gBMQrIxSdPlTYhHql32SVSV9OE04hMC7QEm90/ywtteYHznOLlEbma90ctGaehpoOm/NrH6b1cT6g3RsK4Bf5sfkVfaNbFzgv6v9HP4C4fp+lAXnTd34g15q9G0ebHwlymfyZN6NkXyyeTMz9TBKTYlNuENeWm4uIH4DXHCG8OErwjji/pIH0wjImRTWZ666CmivxGl4z0dhK8Mz3oyLZbsWJaRH40Q/uUwgY4AYzvG2Pen+2aFO5vMcul3LmXZlmUMPzTMzrfsPOl+1v9kPf5WP5P7Jzn2wDEYgqfvfZq65XVEr4my+vOrqVtWt+jtK8pP5zn2L8dIPJpg7OkxJnZOkBvLceFHL2Tlp1ZSF61D/ELb29oIrQvR0NvAcyPPEX9THIDgiiAr3rfitPe/9str6by5k/237mff/9zHob85RM/dPbS+sXWxmrggLPynoKqkX06TfDJJZEsEf6ufw393mL3b9gLg7/TTfEUzne/tRLPONRA7/rDjpPupizgByI/naX1zK4PfGGTgqwM0XNJA/N1x2t/VXvF2TLw4wfBDwww9NETisQQ6rVx858VO7arkxnJ4m7z4O/34wj68YS/+dj8A4Y1hLvnGJTPzi78DHQEA2t/eTvvb2+n7Th+XjF/C0ENDJJ9M4gs7T6uXv/AyuWSO6NYojZc1VmSHp6pM7p0k8WgCqRPa39GOeITdf7QbctC0oYm2328j1BuieVMzAL5mH+v71s++oz7mVF/T5U28+t9eTeInCfbfuh9/h/M/mx6dxhvy1sQoT2r5Ap49PT26a9euBbmvzECG/vv6Z3r1zJEMAOseWMfy65YzuX+SsR1jhDeGCa4IntPfyKayHHvgGEfvOUryiSQbnt3A9pHtXHXZVfjCvgUZMucmckwfmyZ4YZDpoWkejz0OCqFXhYhujdKytYXwFeEFfXKWniCjqjMhev765zn27WMA+ON+otdEiV0Xo2Vry7z/5sDXBzj+veMkHk2Q6Xceq+bXNrP+MSfUE3smCHYH8fjKa+dCneTz4g0vMvroKN23ddP21jbEu3gjPBHZoaobyl3fev6CXCrHvg/tI3hRkMiWiDN83xim8dWNANSvrKd+5fyO9PoafcRviBO/Ic7EngkaVjdAH+y5ZQ+JxxK039hO/IY4gc7AnO53cu8kQw8NMfzQMCMPjxC5OsJrfvga6lrq6H2gl6b/0kSw69x2WHNV2nv2PtBLZiDD8L87I4/j/3oczSotW1tQVQ7fdZjI1RFCl4ZO2+vmp/Oknk4x+ugo4z8fZ+29axERhn4wRPInSSKvjxB5XYTm1zXTsLZhZruG1Q2nvL9Ki10fI/Vsihd//0UO/sVBuj/ZTeza2JI8FmLhLwiuCnLlwJX4l/sX5e+VPjlb39RK+nCaA7ce4MDHD9CytYXOWzqJ/lr0lNtqTmd6lJ1v3cngNwcBqF9TT+cfd9Ly26/0rLHfLftU74rwt/lpf2c77e9sJ5/Nzxxgm9o3xZ737QEgsCJA9DeiRLdGWfYry/A1+Rj6wRCH/voQySeS5Cecg6j1a+rJjmSpi9bRc08PnqCnKsdOzqRlawvRa6Ice/AYBz52gJ3X76T7E910f6y72qWdxMJfICKLFvwTxa6NEbs2xuTeSY7+41H6v9LPyH+MEP21KJpXpg5MIXXC8A8Kr90fTbDxpY34mny0vqmV8MYw0a3RqvV25fL4PHhanGF4/UX1XPHyFTOjgsFvDXL0nqOsu38dy69fTm48R3YoS/zGOM2vayayKYK/7ZXHx1u/dI+ui0dYft1yYm+OMfCNASJbIgCMPTNGdiTLstcvq3KFDgv/ElJ/UT2rPr2K7k92z/R2Iz8e4ae//tOZdQIXBlj+351w+Jp8LP9vy6tV7rwFOgPEb4wTvzFOPpMn8ZPEzHkRy69bzvLrardtAOJ1DkAWHfqrQwx+a5DI6yOs/NRKmq9srmJ1Fv4lyePz4Ak7PWTo0hCr/nIVeJ0hZcMlDUtuqLsQPH4PyzYvjR6xUnq+0kN4Y5iXPv0Sz1z1DPU99bS/o50L/9eF5LN5tr96O6hz0BQAhfh74nR9oItsIsv2X3KWUzxGr7DiT1ac8W3JM7HwL3GBeICuD3dVuwyzALxBLyvev4L4u+McvuswySeT1MWct4NFhNClhesJSOEHZt5WFZ8Q3hieWbe4fD4Hci38xiwyb8hL14dm79DFK/Te33vGbdZ9bd2C1rH0z0QwxlSEhd8Yl7LwG+NSFn5jXMrCb4xLWfiNcSkLvzEuZeE3xqUs/Ma4VMXO8BORm4ERYBVwl6oOlyz7VSAGdAGdwPtVVUXkQaC43g5V/WKl6jPG7SoSfhFZC7Sp6p0i0gNsA24tWaULuE9VsyLyVaAX+DnwPVW9txI1GWNmq1TPvwnYAaCqu0Tk8tKFqvrlkpt+YF9huldEtgEh4AuqmqhQfca4XqXC3wrsLbl9yku5isjVwI9UdQJAVT9YmL8euB14zym2uQm4CSAWi9HX17eghS+2VCplbVgCzoc2zFWlwj8IRADE+fB55sQVRKQLWK+qnz9xmao+IyIrT3XHqno3cDc4F/Cs9W9WPR++HdbaUJsqdbT/MaB4FdF1wHYR8YuIH0BEosAbgTuKG4jIFhHpLUwvA/orVJsxhgr1/Kq6W0SOiMi7gG6ckN+CcwmCzwJfBNLAhsJVaX4MfB/4AxG5FFgLfKQStRljHBV7q09V7zxh1u0ly95yms0+V6l6jDGz2Uk+xriUhd8Yl7LwG+NSFn5jXMrCb4xLWfiNcSkLvzEuZeE3xqUs/Ma4lIXfGJey8BvjUhZ+Y1zKwm+MS1n4jXEpC78xLmXhN8alLPzGuJSF3xiXsvAb41IWfmNcysJvjEtZ+I1xKQu/MS5l4TfGpSz8xriUhd8Ylyor/CJyg4isE5FXicg/F76DzxhTw8r9rr6Equ4Uka8BNwJvP9sGInIzMAKsAu5S1eGSZb8KxIAuoBN4v6rqmbYxxiyscof9zSKyETioqilg6Ewri8haoE1Vvw7cD2w7YZUu4AFV/QzQDPSWsY0xZgGVG/7ngGuAz4jIZUD8LOtvAnYAqOou4PLShar6ZVXNFm76gX1n28YYs7DKGvar6tPA0wAiMg586SybtAJ7S27XnWolEbka+JGqTohIWdsYYxZGWeEXkdtwhuJXAuuAceDWM2wyCEQK2wqQOcV9dgHrVfXz5W5TWHYTcBNALBajr6+vnCYsWalUytqwBJwPbZgzVT3rD3AtzkuEbxduv/Ms668BPl2Y7gVuwxne+wvzosDNgJxpm7PVtWbNGq11Dz/8cLVLmDdrw9IAbNcy8lz8Kfdo/0rgC8D9ItIINJ1lh7JbRI4U3hLsBu4AbgEE+CzwRSANbHA6eX6sqv90im2MMRVSbvj/BuhS1QMishr4f2fbQFXvPGHW7SXL3lLmNsaYCin3aL8CV4vI54CrgP9buZKMMYuh3PB/ABgD7gYSwAcrVpExZlGUO+w/pKoPFqZfFJG3VaogY8ziKLfnbzzhdmihCzHGLK5ye/6jInIvcBC4AHjwzKsbY5a6cs/w+zcR+U9gNc5ZeJsqWpUxpuJOG34R+QzO+fV64iKgHfh+BesyxlTYmXr+h1T1w6daICJXVKgeY8wiOe0BP1V95AzLnqhMOcaYxWKX8TLGpSz8xriUhd8Yl7LwG+NSFn5jXMrCb4xLWfiNcSkLvzEuZeE3xqUs/Ma4lIXfGJey8BvjUhZ+Y1zKwm+MS1n4jXEpC78xLmXhN8alLPzGuFRVw1/4Km5jTBWUe93+ORORm4ERYBVwl6oOlyxbCfw68Fbg6pL59+BcHRjgJVX9VKXqM8btKhJ+EVkLtKnqnSLSA2wDbi0uV9X9wN+LyPUnbPq4qt5biZqMMbNVquffBOwAUNVdInJ5mdt1icj7gSjwJVU9WqH6jHG9SoW/FeebfYrqytlIVT8JICIrgC8Bv3PiOiJyE3ATQCwWo6+vb761VlUqlbI2LAHnQxvmqlLhHwQiMHNQLzOXjVX1ZRE55ZeBqurdOF8VTk9Pj27evHl+lVZZX18f1obqOx/aMFeVOtr/GLChML0O2C4ifhHxn24DEblURDYXpuuAVIVqM8ZQoZ5fVXeLyBEReRfQDdwB3IJzJP+zhYOAvwxcICLvAB4HdgHXiEgMWAN8rBK1GWMcFXurT1XvPGHW7SXLduGE/b7TrWOMqSw7w88Yl7LwG+NSFn5jXMrCb4xLWfiNcSkLvzEuZeE3xqUs/Ma4lIXfGJey8BvjUhZ+Y1zKwm+MS1n4jXEpC78xLmXhN8alLPzGuJSF3xiXsvAb41IVu4xXrfnE959HEDoiQToi9c5Pc5DWxgAej32rmDn/WPgLnj44yu7+MSanc7Pm13mFeHM98eYgnYWdQry4g2iupyMSpClY1tcSmBqRzeUZHEtzNDHF0cQk/YkpjoxOMTA2RSabJ59Xcqrk8kpelWzO+Z3LKznFWZ4vnVf4ndeSbSHg89AWDtAWDtIWDrI8HKCtafZ0uN5Hpb7S0sJf8N33XoWqkpic5vDoJEdHpziSmOTI6BRHRic5MjrJU/uH6U9OkcvrrG2bgr6ZHUE8Uk9npJ5YYwCAbOHBzueV7KwH3/nZuy/DjsyuV5ad+GQpPLmagnUzo5J4s/M7VsOjknxeGZ2cZiiVZmg8w/B4hqFUmsTkNME6L6GAj1DAR2PAS8jvTDcFi/N8BHyecwpFLq8Mjk05wR51wn00McVP90xxx87H6U9MMZCc4oSHmAa/l/ZwEL/Pg9cjeD2CR2Rm2ufxEPAJHo/gFU5a7vUIXikud35PTecYHJviF4Mp/nPPccamsifV6+wggrSFAywPBws7h8ArO4jCjqMxMPcoW/hLiAiRBj+RBj+9Hc2nXCeby3MslebI6CSHR6c4WtgxHEk4O4lnD40yMjE9p7/r2bfntE+W4vzk1DQTmZNHJW3h4KwdT/HlSrzZ2QlVsucolcsroxOFEI9nGEplGB5Pl0xnGBpPz0yPTGROCthceD1CyO+lsbiTCDo7heKOojHg7EAy2TxHk87j1J+YYmAsfdLOu77OS7M/z+q4j6tWt9LRHKS92RnhxQv/y3Cw8v/HiUyWwWSageQUg2Ozfw8kp3jhSJKHk4MnPQ8AQn7vnP+eqM7jEaiynp4e3bVrV7XLOMlkJsfxVLpkLw8+j2dmujTcjz36CFu2bDnrfRZHJUcKvVVxh+PsfJxRSn9iiuwJT+wGv3dmpNBReEIvbwqSVyWTzTOdc34y2TyZnM5MvzIvXzJPyZywfDqXZ2hsgvFpThvmSEMd0ZCflpCfllCAaKOf1pCfaMhPtDHgTDc6t5vr60hn84yns4yns6TSOcbTWcamnNvjmSypwrLxdI5UOktq6tTzx9NZfF6ho7me9kKIOyJB2puDM/M6mp0d5COPPFIz39iTSmdndgjFncVAMs3H39i7Q1U3nP0eHNbzV0C938sF0Yay1i23NykdlazrCJ9ynVxeOV4YlRxNFF+uTBVuT/Ji/xjHxtKn/Rt+nwe/10OdV/D7PNR5ndvF6eL8pqBv1vzkUIZXrel2At4YoKUQ7JZGP8sa/NR55/amUsDnJbwAx1GKHdtijHwWU2PAR2OskYtijbPmf3yO92PhP494PTLzGnD9adZJZ3MMj2fwemRWgH0eOeeQON9z13PuhVfI+Rb6hWbhd5mAz0u8ub7aZZglwE7yMcalLPzGuJSF3xiXsvAb41I1/T6/iIzhfNV3LWsFjle7iHmyNiwNParaVO7KtX60f9dcTmpYikRku7Wh+s6XNsxlfRv2G+NSFn5jXKrWw393tQtYANaGpcF1bajpA37GmHNX6z2/qSI5D06ePx/acK5q9mi/iNwMjACrgLtUdbjKJc2ZiDwIFOveoapfrGY95RARD/BW4Drgs8DjIrISeBtwGJhU1W9VscSzOk0b3gW8tmS1j6vq4SqUVxYRiQA3AgPAG4BPAY3ArwDjwC9U9X+f6T5qMvwishZoU9U7RaQH2AbcWuWyzsX3VPXeahcxF6qaB74mIquBYq/5UWCbqiZF5B4R+XdVHa1elWd2mjagqu+uXlVzdgnwH6r6MxE5DNwEXAD8nqrmReSbIvJ/VPXkK38U1OqwfxOwA0BVdwGXV7ecc9YrIttE5KMicupLB9WGNlVNFqZ3A5dVs5hzNCEi/0NEbhORN1S7mLNR1SdU9WeFmy3Az4C6wo4NnBHlBWe6j5rs+XHOxtpbcrsmr6Cpqh8EEJH1wO3Ae6pb0Tnzl0wngFi1CjlXqno/zLwk+IaI/EJV91W5rLMSkQaczvBzOEP+ouLjcOB029Zqzz8IRGDmgE2muuXMj6o+A6ysdh3zUPr/X4bzOrQmFXrOh4FfqnYtZ1N47m8DPgEcBUpP7T3r41Cr4X8MKJ6KuQ6Y02mNS4GIbBGR3sL0MqC/yiXNR3/Jy5aLgWerWcxcieO9JbNWAS9Uq545uBG4V1WHVTUDZESkeCXPZTgHYE+rJof9qrpbRI4UjtB2A3dUt6Jz8gzwByJyKbAW+EiV6ylLYZj5mzg9Y1BE/DhHmrcVDjz9qOT1/5J0YhtwXrakROS3gE5gj6r+vIolnpWIXAv8HvDawruVaeAvgQ+LSAr4hzMd7AM7yccY16rVYb8xZp4s/Ma4lIXfGJey8BvjUhZ+Y1zKwm8qTkQ2i8hfVbsOM5uF31ScqvZRu5+/OG9Z+I1xqZo8w89UjojcAowCFwKPAm8GvDiforwYOKiqfy8ijcAtOB+wagOeUtWnRCSG8/HS3cBFwD2qOlS47zcA64E3Atcs9TMBz3cWfjNDRF4P1Kvq3xY+NHIf8I/A64rXHRCRb4vIt4E/Ab6pqj8vrPtdEfkd4C+Aj6jqoIh0Au3AEM4o8zlV/aGI5IErgB8udhvNKyz8plTxfP034fT2LxXm50vW2YszKngNcBuAqqqITALNwIWqOliYf5hXPlySV9Xih5cGqcGP/Z5v7DW/KfUCMKaq31HVf6EQbkqudoMzlN+Hc/GINTDz0dJ6nM+QHxeRi4sri0h4Eeo258A+2GNmFEL8SZxPub2ME3CAP8L5jHs38IKq3lcI9S04O4I24ElVfUJEuoBPA7/AuZrMPwNdwFeB9+F83PfPgRzwp6o6tjitMyey8JszEpHNwGtV9c+qXYtZWDbsN6clIiHgSmCdiFxU7XrMwrKe3xiXsp7fGJey8BvjUhZ+Y1zKwm+MS1n4jXEpC78xLvX/Abv+w+NPApbDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAAC4CAYAAAAhS0pZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAHeRJREFUeJzt3Xt8XHW57/HPk5lMrm1ubS5Nk7TpJW16pRcELNqCpVjRDRyPvACPR0VxH0E2CCoXOSIIiuwt6uZ4QVR0o2JBtgqIArYV1AJNKaUtbdqmSdOkaZI2NO0015l5zh8zSYeSe2ZlMtPn/XrllbmsWfOsNt9Zl/mtZ4mqYoyJXwnRLsAY4ywLuTFxzkJuTJyzkBsT5yzkxsQ5C7kxcc5Cbkycs5AbE+cs5MbEOQu5MXHOHe0ChiozM1NnzpwZ7TJG5eTJk6SlpUW7jFGJ9WWI9foBtmzZckRVJw91ekdDLiITgXWqevFpj08HrgbqgXZVfXyweeXl5VFRUeFMoWNk48aNrFy5MtpljEqsL0Os1w8gIgeGM71jm+siIsD/AJr6ePqrwPdV9efAhSKS6VQdxpzpnNwn/zDwHBDo47k8VT0eur0HWOxgHcac0RwJuYiUA8dU9XA/k3jCbrcCQ96/MMYMj1P75JcACSJyHrBARG4luHneFnq+K2zaLGB3XzMRkWuBawEmT57Mxo0bHSp3bHi9XluGKBuwfgW2A08AbcC3gMSxqsxBquroD/Bo6HcKkBC6/QiQEXZ74mDzmT17tsa6DRs2RLuEUYv1Zeirfn+3Xw//5rBWLKvQDWzQDWzQyusq1d/hH/sChwCo0GFkcCy/Qvs28DKwDrgHuElE6oEX9dT+uTFj7sh/H2HXlbtImZ3CrB/OIv8T+bhSXdEuK2IcD7mqfjL0+wthjx0A7nL6vY3pS0dtB3XfqyO5JJmpN0xl0qWTWPDsArIvzkYSpHe65qeaadvTRsmtJVGsdvRsxJs5YxzffBzuhldKX6Hue3V01HQAkJCYQM7anHcEHKDl+RZq7qyhrbKtr9nFDAu5OSNUfbmK189+HV6DopuKOKf6HGZ+Z+ARlNPvnk5CagJVt1SNUZXOsJCbuOQ/6af+B/V01AbX1jkfyWHGgzNgHcx4YAbJRcmDzsOT66HkjhKOPnOUlhdanC7ZMRbyUTqx9QR7v7C355sEE2WdDZ3sv2M/m4o3sfe6vTQ/2QxA5opMim4sgtThzW/qv00leXoyVV+sIuDra1zX+BczJ6iMN52HOkmakoQkCE2PN5FUnETxl4qjXdYZS1XZ87k9HH70MOpTJl06ialfnErGezNGNd+EpARmPTQL39u+d+2zxwoL+TD5O/xU31bNoR8dYmnFUtIWppF5QSb7b91P+uJ0sldnR7vEM5KIQAIUXFvA1BunkjpzmKvsAeSszYnYvKLBQj4M3u1edl29i5PbTzLluikkT09GRJjzszm07WrjrSveYmnFUlJKU6Jd6hkh0B2g6pYqcq/MJeOcDMp+VObo+x38zkH8bX6mfXWao+8TabZPPkR1369jy7ItdDV1seDZBcx+aHbvgAlXmov5v58PwK6rd8Xl/rmqov7xs1zdx7rZvnY79d+v59iGY2Pynt43vRy45wDt+9vH5P0ixdbkQ9RZ30n2mmzKHinDk+t51/MppSnM+908EnMSg5uOMUBV8bX66G7sputwF8mlySQXJdNe3c6Bbxygq7GLrsPBn+6mbsp/Uw7jYMu1bV8bOz68g/aqdsp+XkbBJwvG5H1L7yul+Ylm9n9lP/OemDcm7xkJFvIBND/VTOLkRDLPz2T6vdMRlwwY4KxVWb23vdu9pC9IH4sy+9XV3EXrS629Qc14XwbZq7PpqOtg64qtdB3uQjtPrZ1nfm8mU2+YSqAzQMufW/DkefDke0hfkI4n30PKzBR4G5qebCIxK5GsC7MGeHdntO1t4/VzXgdg0YuLyHzf2LUiSJqSRPFXiqn5Wg3HXj5G5vmx0QbBQt4H3wkf+27cx+GfHWbS5ZPIPD+TBPfQ92waftZA5WcrWfjnhVE7EOd908uba96k63DohD+BElcJ2auzScxKJPP9mb0h7vmdWh48WJU2J43z6s/re8Yb4OADBznx2gkKry+k9P7SMR3nnVKaQt7/yqPw+sKIHlwbqqJbimj4SQP7btrH0s1LY2KrzUJ+mtZXWtn18V10VHdQfEcx0742bdjzyL0il7rv1kX1QFzVl6rABYs2LCJ1TiqJkxJ7P6hcaS7m/mLuyGYssHjDYvbfvp/679XT8pcW5vxiDhnnju6rqoGoXzlw7wEKrikgqTCJWd+d5dh7DcaV6qLskTIS0hJiIuBgB97eofWVVrau2Ir6lMUbF1P6jVISEof/TxR+IG7HpTvwn/RHutR+9Rz0m/uruSz55xKyVmaRlJ80rC2RwbhSXcz67iwWrV9EoCvA1hVbHRvf7fP62HHZDmq+VkPT4311Eht72WuyyVwRG5vqYCEHINAVHMk08eyJlN5XyvJty0e9v5VSmkL54+Wc3HmS3df02RMj4hp+1sD2D20n0BXAM8lDcvHgQzdHI2tVFsvfXE7ZT8tILQtuOncf7Y7Y/Dtqg8cOjv7pKLMemkXRzUURm/doaUDZc90eau6piXYpgzqjQ66qNPy8gVdnvUpnfSeSIBR/uRh3RmT2YrIvymbmd2aS+7HciMyvP6pK7f21VF5TifoV7R67r7rcE929R7dPvHGCTSWbOHDfgVEPAfXu8LLl7C10VHew8NmFFF5XGIlyI0YShO6j3dR+s5aOuo5olzMgp3q8TRCRO0TkKhF5WERmnvb8ShH5rYg8EvpZ5EQdA+k+2s3Oj+6k8tOVju4zT/23qUy+PNTCzoGvVzWgVN1cxf5b95N7ZS4Lnl6AKy06DQ+Si5LJuSSH6juqR70Jn1yUzIRlE1iyaQnZa8bnKMIZ989AA0r1bdXRLmVATq3JJwMvqeqvgV8Cn+tjmq+o6mdCP9scqqNPLS+0sHnhZo4+fZTSb5ey6K+LSCpMcvQ9m37bBFdDe1Vkk151SxV1D9ZReEMhcx+bS4InehtniTmJzHt8HuWPl9O+t52KxRXUPVQ35NerKocePoS/3Y87w83CZxaSVj5+L4SQXJJM0ReLaHyskeOvjd/mRo78RajqflV9OXR3OvDSaZN0AleJyO0icpUTNQzk8C8O485ws+TVJRR/qXhMTjyYsHwC+GDHZZE9EJf/v/Mp/XYpM787c9ycQJF7RS7Ldywn88JMuhuHto/ub/ez6+pd7PncHhp/2ehwhZFTfFsxiXmJ7L9tf7RL6ZdjX6GJSD5wK8G+l/83/DlV3QRsCk33oIgcDPtQcNzsH85GXDLm3+9yJ5y89SS7P72b8sfLR/wVTPfRbpoeb2LK56eQviid9EXRHXTTl6SCJBY8vaC3637Liy10Hugk/9P571ruzsOd7Lh0BydeO0Hpt0opuHZsRrBFgnuCm/JflZNc6uxBztEQp8dZi0gJcKOq3tTP82uAhar6QB/PhbdkXrpu3TpHa3Wa1+sl/el0eJjgUl05gpk0AV8GDgE/Bcb4gLPX6yU9fQQfKvcCLwLnAjdzanjsfuA24DhwB7AiImX2a8T1D1UAxw9nr1q1aouqLhvyC4bT2nWoP8ASYGrodjawnuAFFTyhx67j1AfM54G1g80zXloyBwIB3XnlTq2+u3rYr/e+5dV/Fv1TX5rwkrZsaIl8gUMw0pbMAX9Aax+s1b8l/01fzn5ZG3/bqKqqJytP6msLX9Pjrx+PYJX9c6qltK/dp9su3qY136xxZP7hGCctmeuBj4vIUWApcDtwAyDAA0AtcHnogohpBC+ndEYQEeY+Nrd3/1lVh7TZfvzV47y59k0kUVj8t8VMOGuC06VGlCQIRTcWkX1xNrs/sZu3rngLd6ab7IuyWbZ12bg5njBSrmQX4hFq76sl/5P5JOU7eyB3OBwJuao2Av8Ruvto6PcrYc8/7cT7xoqeP+i3N77NgXsOMP8P83GnD/xf0XGwg8ScRBY+t5CUGbF7vnranDTO+udZND3eRNbq4AkusR7wHjP+fQab522m5s4ayn7i7Lntw3FGD4aJNu1Wjm08RuWnKvs9B72nEWHuR3NZvn15TAe8R4I7gfyPv/sAXKxLnZVK4fWFNPy0Ae82b7TL6WUhj6Ls1dmUfquU5iebqb2/9l3P132/jldnvUrrP1qBYL8xM76V3FmCO9vNgW8M6xLijrKz0KKs6JYivK97qb69mvTF6eRcnIOqUn1nNbX31jLpskmkLx1/X5GZviVmJbLgjwtImzd+BvHYqiHKRISyR8pIW5BG8xPNBHwB9nxuD7X31lLw2QLmPTEPV3L8XJfrTJBxXgbuDDcBX4BAd/TbOFvIxwFXmovF6xdT9kgZzb9tpuEnDZR8tYTZPw4O2jGxp7ulm4rFFdQ/VB/R+fq8vmG/xjbXx4nEnOCFsHOvysWT74lKayUTOe4sN0mFSRy4+wD5n8jv/f8djAaUzvpO2qva6ajqoH1fO+1V7cEe8udkcOyvw29aaSEfZ0TEAh4HRIQZ/zGDikUV1NxVw6z/PNXNJtAdoKOmg/aqdtr3BcOcvTab7NXZeN/wsmXpllPzcQvJ05PpPhI8B2DieROHXYuF3BiHpM9PZ8q1U6h/qJ70JekUfKqA7pZu/pH7Dwg7RykhNYGkkiSyV2f3XiM9ZWYKKTNSSCp6Z1cfz+R3dwoejIXcGAdNu3saR35/hOOvHKfgUwW4s9xMu3MaydOSSZ6RTMqMFDz5nt4xA+50N4X/GtkGGRZyYxzkmezh3EPn9oZYREbUHHQ07Oi6MQ6L9sg+C7kxcc5Cbkycs5AbE+cs5MbEOQu5MXHOka/QRGQCwU4w1cBK4Nuqui/s+RzgemAfkKGqP3CiDmNM9Pqu3wz8WlV/BRSKSPSuYGdMnItW3/WFqro3dPsN4L1O1GGMiVLfdYKdW3u0AtP6mUd4S2Y2btwY8TrHktfrtWWIslivf0SG09p1JD9ACfDgaY89E3b7CuATg80nXloyx7pYX4ZYr191+C2Znbrg4RIRmRq6ewJYJCIeEelZg28VkTmh22cBf3eiDmOMc5vrg/VdfxC4UUSqgFpVHb8XkjImxkWr73oL795PN8Y4wAbDGBPnLOTGxDkLuTFxzkJuTJyzkBsT5yzkxsS5EYVcRM6OdCHGGGf0+z25iHweuAw4/Zq6AuQDCxysyxgTIQMNhvGq6uq+nhCRyxyqxxgTYf2GXFV/GX5fRLIInmyyX1X/2+nCjDGRMaRhraFTPlcDVcAMEfmDqj7maGXGmIgY6tj1YlX9nz13ROQ+h+oxxkTYUI+u7zntfjWAiEyPbDnGmEgb6pp8mYgUAwGCHwzlIjIZKAc+7lRxxpjRG2rI/byzT1tP/7bTv14zxowzQw35zwk2fsgGDgN3quoBEdnc18QikglcAzQCa4B7VHVP2PMrgf9DsGsMwH+q6rYRLYExZkBDDfm/Al9Q1WYRySPY8OE6VW3rZ/q5wPOqul1E6gk2Y7zltGm+oqo1IynaGDN0Qw35NlVthmDXFxGpBBCRJFXtPH1iVd0UdjeHYNvlcJ3AVaFLutZosD+7McYBQw358VBjxgDBYa0+EZkCfIDgxRP6JCKpwPnAl8MfD30IbApN86CIHNRTfdrDX28tmceZWF+GWK9/JCTY4XWQiUReBvYSDHi4YlW9sJ/XCMH9+B+Gerr1N+81BC+28MBANZSVlWllZeWgtY5nGzduZOXKldEuY1RifRlivX4AEdmiqsuGOv1Q1+Q3qWpFH2820OWNrgEe7Ql4TztmVe0SkeuAH4R6SM8Adg61YGPM8Awp5H0FPPT43r4eF5HLgSuBFaH97k5ObQk8ANQCl4vIRCANeG7YlRtjhsSplsxPAU8N8PzTTryvMebdrDOMMXHOQm5MnLOQGxPnLOTGxDkLuTFxzkJuTJyzkBsT5yzkxsQ5C7kxcc5Cbkycs5AbE+cs5MbEOQu5MXHOQm5MnLOQGxPnHDmffAgtmXOA64F9QIaq/sCJOowxzq3Je1oyPwb8jFAzxjA3A79W1V8BhYO0kTLGjIJTnWEGa8m8MKx11BvAewm2hzLGRJgjIe/RX0tmwBN2uxWY1s/rrSXzOBPryxDr9Y+EYyEPtWS+Cfh6Hxdg6Aq7nUVw3/1dVPVh4GEItmSO9Va68dAOONaXIdbrHwknj66/qyVzT1tmYKuIzAndPgv4u4N1GHNGc+ro+mAtmR8EbhSRKqBWVfc7UYcxJnotmVsIXjTRGOMwGwxjTJyzkBsT5yzkxsQ5C7kxcc5Cbkycs5AbE+cs5MbEOQu5MXHOQm5MnLOQGxPnLOTGxDkLuTFxzkJuTJw740IeCCi1R9uiXYYxY2ZMQh7qEjMubNp/lPc9sIErfryJp16vo73LH+2SjHGUYyEXkQQR+biI/B44r4/np4nIsyLySOjnAqdqCTcrL50vrSmj8XgHX1y3jbPvfZHbntrOGwePoapjUYIxY8qxHm+qGgAeE5GZBDvC9OUBVd3oVA19yZ2QzHWrZvL5lTN4rbqFdRV1/H5rPb95rZbZeel8bFkRl55VyKT0pLEsy4wjPn+AxhOd+P2KLxAgoIovoPgDSiDAqcf8il+Dj/sD2vtY+PT+gKIK8wszmJ2XTjQ2ah3t1joIP/BBEVkOnAR+FPpgGBMiwntKc3hPaQ53faScZ95sYF3FQb7x7C6+9dxuPjA3j48tn8r7Zk3G7TrjDl2ccbr9ATZVHeW5HQ38ZWcjLSe7Bn/RMJXkpHJReR6ry/NZWpKFK2FsAi9Ob6KKyF3Ai6rab7NGEbkROKqq/3Xa4+EtmZeuW7fOyVIBqD8R4OX6bv5xyMeJLshMElYUullR6CY/bXRh93q9pKenR6jS6Ij1ZQiv3xdQdh71U3HYz+tNPk52Q7ILFue6mJPtIjEBEkRIEN7x4+q93d9zoccJPhZQ2N3iZ2uTn7eO+vEpTPDA4sluluS5mJfjwuMaeuBXrVq1RVWXDXX68RLyMuAGVb2uv2nKysq0srLSgQr71uULsH53E09UHGRDZRMBhbOnZfOx5UWsXZBPqmf4G0Hx0A441pfh+b9uIKGgnD9tb+CFXY2c6PAxIcnNB8rzWLuggPNnTSI50eXY+5/o6OZve5p54a1G1u9u4kSHj5REF++bPYnV5flcOCeXrDTPgPMQkWGFfEw310UkAUhS1XYR+QzwX6Ge7DOAnWNZy2A87gQunp/PxfPzaTzewe9er+OJijpueWIbd/1xJ5csLOBjy4s4qyhzxPtZgYDS5Q/Q6QvQ5QvQ5Q8QCChTs1Kisu8Wrzq6/WysbA5uim9vo8NfwcRkN2vm5bN2QT7vnTmJJLdzwQ43ITmRSxZO4ZKFU+jyBXi1+ijP72zkhbca+cvORlwJwvJpWawuz+ei8jyKslNH/Z6OrclDV0/5EPAJggF+nuAlk96vqteLyNlAOeAjGPJvqmq/O0JjvSbvi6qyueZt1lUc5Nk3G2jv9jMzN51lJVl0+QJ0+oNhDYbWH3Y7GODj3nbEnRh63E+3v+9/+/yJyayaM5kL5uTx3pk5I9pqcMpo1uQd3X6217eyuaaFipq3OeLtJG9iMgUZyeRnhH5PTOm9P5o1aluXjw27m/nTjgY27G6irctPVmoiC7KVay5awrmlOXjc4+dYi6qyvb61N/CVjScAmFswMbQfn8e8KRMRkWGvyR3fXI+U8RDycN5OH89sO8QTW+qobWkjyZ2Ax51AktsV/O3quR/87XEn0NLcREnRFDwuF0mJCXjCpumZrsuv/HPfEV7eewRvpw+PO4FzS3O4cG4uq8pyI/LJPhrDCXlrWzdbalvYXPM2FTUtbDvYSpc/eGx1xuQ0pmSm0Hi8g8OtHRzv8L3r9VmpieRnnAp9wcSeD4MU8kOPpSed+gD0dvpYv7uJ57Y3sKGyiY7uADlpHtbMz2ft/ALOKc3m7y+/FBO7GzVHTvLCW8HAbz7QgioUZqawujyPr//LfAv5eDWcgHT5AmyuaeGvu5pYv7uRmtAovdl56ayak8uFc/JYUpw55kf+B1qG+mPtVNS08Fp1cE3dszZyJwjzCzNYPi2L5dOyWVqSRc5pX1Ge7PRxOBT4htYODre2h3539D5+tI8j3hOS3ORnJJOZmsi2ula6fAEmT0jig/Pz+eD8As6env2Oo9ixeEzhiLeT9buaeP6tRl7e28yee9dayMer0fyB7W/2sn53E+t3N/FadQu+gJKRksj7Z0/mwrm5vH/2ZDJTBz5gEwk9yxAIKJWNJ6ioObWmPtTaAUB6kpslJVksL8li2bRsFhdlkuIZ/T5vR7efpuOdNLS2c/h4x6kPgdYOmr2dLCjM4EMLC1hS3P/XU7EY8nBtXT7SkhLH74E3M3Klk9MpnZzOZ84v5XhHN3/fe4S/7mpiY2UTf9x2iASBpSVZXDAnjwvm5I564EW3P0Bbl5/2Lj9tXT7auvy0tnfzTFUXv6h+jYoDb3MitImdOyGJ5dOzuTYU6jn5ExzZwkhOdFGck0pxTnR3WaJpJMdnLOQxaGJyImsXFLB2QQGBgLKt7ljvWv7+P+/m/j/vpjAzhQvm5FKWP4GObj9tXT0/vt7wngy73fN4z/2efee+zMxt55KFU3o3v+3bgPHNQh7jEhKEs4qzOKs4i5svKqOhtZ0Nu5tZv7uJJ7fU0d596gQcjyuB1CQXqYkuUjwuUj1uUj0uJqV7SPWkkupxkepxkRJ6PDVsmhSPi/QkNy37t/Phi94fxSU2w2UhjzMFGSlc9Z5irnpPMR3dwU3sFE8w2JHYhN5YZ2vsWGMhj2PJiS5HR2+Z2DB+RgMYYxxhITcmzlnIjYlzFnJj4lzMjHgTkRNAbA95g0nAkWgXMUqxvgyxXj9AmapOGOrEsXR0vXI4Q/nGIxGpsGWIrlivH4LLMJzpbXPdmDhnITcmzsVSyB+OdgERYMsQfbFePwxzGWLmwJsxZmRiaU1uomg8XQVnpOJhGUYiJo6ui8j1wNtAKfD/VLUlyiUNm4g8BfTUvUVVfxjNeoYi1HjzKuCjwAPAP0RkOnA1UA+0q+rjUSxxUP0swyeBFWGTfU1V66NQ3qBEJBO4BmgE1gD3AOnAhQSvV7BXVV8YaB7jPuQiMgfIU9WHQq2bbwLujHJZI/FHVX002kUMRz9XwfkqcJOqHheRn4jIn1X1WPSqHFh/V/JR1c9Er6phmQs8r6rbRaSe4HUIioArVTUgIr8RkfWq2u9F/WJhc/18YAuAqlYCS6JbzojNE5GbROSrIpIR7WJGIU9Vj4du7wEWR7OYEWoTkc+LyF0isibaxQxEVTep6vbQ3RxgO5AYdrWhFoKh79e4X5MTHKFUFXY/MVqFjIaqfglARM4C/h34bHQrGrHwRnKtwORoFTJSqroOejflfy0ie1V1f5TLGlCoxfn5wHcIbqr36Pk/qOnvtbGwJm8CMqH3wEnkL1I1hlR1KzA92nWMQvi/fxbBfcWYFFobbgCWRruWgYT+7m8Cvg40AOFDWgf9P4iFkL8M9AxDLAeGNaRvPBCRVSIyL3Q7Czgc5ZJG43DY7sYs4I1oFjNcEhR+Oa5SYFe06hmia4BHVbUldAGSLhHp6QaSRfAgaL/G/ea6qu4RkUOhI6LTgO9Ft6IR2Qp8WkTmA3OA26Ncz5CEXQVnKZAsIh6CR3dvCh0EejFs/3xcOn0ZCO5ueEXkEqAQ2KeqO6JY4oBE5HLgSmBF6BvATuBbwFdExAs8MtBBN7DBMMbEvVjYXDfGjIKF3Jg4ZyE3Js5ZyI2JcxZyY+KchdxEjIisFJFvR7sO804WchMxqrqR2D23IG5ZyI2Jc+N+xJtxhojcABwDSoCXgMsAF8Ez/mYBtar6YxFJB24geJJQHvCqqr4qIpMJnva4B5gB/ERVj4bmvQY4C/gIcPF4HxUX7yzkZyARuQBIUdXvh05++CXwU+B9Pee8i8iTIvIk8EXgN6q6IzTtH0TkX4BvArerapOIFAL5wFGCW4fbVPUvIhIAzgX+MtbLaE6xkJ+ZesaiX0pw7X0g9HggbJoqgmv5RcBdAKqqItIOZAAlqtoUeryeUydJBFS15wScJmLwVNR4Y/vkZ6ZdwAlV/b2q/o5QiAnrnEJwE3w/wSYFs6H3lMcUgucwHxGRWT0Ti8jEMajbjICdoHIGCoX1boJnZNURDDLAvxI8v3oasEtVfxkK7w0EA58HvKKqm0SkGLgP2EuwO8lvgWLgF8AXCJ6Cei/gB25T1RNjs3TmdBZyAwS/4wZWqOo3ol2LiSzbXDeISBpwHlAuIjOiXY+JLFuTGxPnbE1uTJyzkBsT5yzkxsQ5C7kxcc5Cbkycs5AbE+f+PwpqQxCy528dAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, 0.1, num_epochs, lang, device, first_train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, lr, num_epochs, lang, device, first_train=False)\n",
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder2{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder2{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": "400"
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
